'''网页保存脚本.保存后图片与css也一起保存,并保证位置可用.
  用法是:新建一个HtmlDown类,构造方法参数:
    website:要下载的页面的主站地址,如:https://ameblo.jp
      结尾不要有/符合,这个参数用于一些缺省的地址的补完.
    url:要下载的地址
    outputFile:要保存的网页的文件名,不含路径,路径在dir参数设定.
    dir:表示要保存的网页的文件夹,网页会保存在该文件夹下,相关图像会
      保存在该文件夹下的img文件夹中,相关css则直接保存在该文件夹下.
      若保存多个网页,其中有公用的图片,也能在img文件夹中公用.
      图片地址会被更名成src的md5+对应扩展名,放在img文件夹下.
    
  使用时用main()一键搞定.
  
  Last updated @ 2019/8/16
  By setycyas
'''

import sys
import os
import hashlib

import requests
import lxml.html
from lxml import etree

class HtmlDown:

  def __init__(self, website, url, outputFile, 
    dir = sys.path[0], encoding = 'utf-8'):
    self.session = requests.Session()
    self.html = ''
    self.htmlTemplate = ''
    self.website = website
    self.url = url
    self.outputFile = outputFile
    self.dir = dir
    self.encoding = encoding
    return
    
  ## 访问夜歌页面,获得其url    
  def getHtmlByUrl(self, url):
    resp = self.session.get(url)
    self.html = resp.content.decode(self.encoding,'ignore')
    return
     
  ## 从本地文件获取html,测试用
  def _getHtmlByFile(self, filePath):
    with open(filePath,'rb') as f:
      self.html = f.read().decode(self.encoding,'ignore')
    return
    
  ## 对获取的html文本进行额外修正,指定修正方法,没有就跳过,不常用
  def modifyHtml(self, modifyFunction = None):
    if modifyFunction:
      self.html = modifyFunction(self.html)
    return
        
  def _modifyFunctionTest1(self, html):
    htmlTree = lxml.html.fromstring(html)
    targetDiv = htmlTree.xpath('//div[@id="entryBody"]')[0]
    return lxml.html.tostring(targetDiv,self.encoding).decode(self.encoding)
        
  ## 获取一个对象的md5
  def get_md5(self, src):
    myMd5= hashlib.md5()
    myMd5.update(src.encode('utf-8'))
    return myMd5.hexdigest()
    
  ## 保存网页到target以及img,css,最后修改网页内容保证img,css可用.
  def saveHtml(self, isSilence = False):
    if not os.path.isdir(self.dir+'\\img'):
      os.mkdir(self.dir+'\\img')
    tree = lxml.html.fromstring(self.html)
    ## 分析img,并保存图片,修改链接
    imgs = tree.xpath('//img')
    for i in range(len(imgs)):
      img = imgs[i]
      src = img.get('src')
      if not src.startswith('http'):
        src = self.website+src
      newSrc = self.get_md5(src)+os.path.splitext(src)[1]
      img.set('src','img/'+newSrc)
      if os.path.exists(self.dir+'img\\'+newSrc):
        if not isSilence:
          print(src,'has been downloaded')
        continue
      ## 获取图片
      try:
        resp = self.session.get(src)
        with open(self.dir+'\\img\\'+newSrc,'wb') as f:
          f.write(resp.content)
        if not isSilence:
          print('src to newSrc:')
          print(src)
          print(newSrc)   
      except:
        if not isSilence:
          print(src,'fail')
    ## 分析css,并保存图片,修改链接
    links = tree.xpath('//link')
    for i in range(len(links)):
      link = links[i]
      href = link.get('href')
      if not href.startswith('http'):
        href = self.website+href
      if not href.endswith('.css'):
        continue
      newCss = href.split('/')[-1]
      link.set('href',newCss)
      ## 保存css
      try:
        resp = self.session.get(href)
        with open(self.dir+'\\'+newCss,'wb') as f:
          f.write(resp.content)
        if not isSilence:
          print('css to newCss:')
          print(href)
          print(newCss)   
      except:
        if not isSilence:
          print(href,'fail')
    ## 保存网页
    target = self.dir+'//'+self.outputFile
    with open(target, 'wb') as f:
      f.write(lxml.html.tostring(tree, self.encoding))
    if not isSilence:
      print(target,'saved!')
    return
  
  ## 一键完成任务的主函数  
  def main(self, isSilence = False):
    self.getHtmlByUrl(self.url)
    self.saveHtml(isSilence)
    return
    

## 使用实例,简单,下载广东象棋网的一个新闻页面        
if __name__ == '__main__':
  website = r'http://www.squarecn.com'
  url = r'http://www.squarecn.com/article/bc5739_a5884.aspx'
  outputFile = r'FF1低等级极限攻略（决战篇）.html'
  hd = HtmlDown(website, url, outputFile, dir = sys.path[0],
    encoding = 'utf8') 
  hd.main()